%% @author Eric Florenzano <floguy@gmail.com>
%%   [http://www.eflorenzano.com/]
%% @doc This module handles all of the URL segmentation logic for erlysocial.
-module(es_urls).

-export([test/0, harvest_urls/0, harvest_urls/2, start/0, get_more_urls/1]).

-define(NUM_REQUEST_THREADS, 20).
-define(DEFAULT_TIMEOUT, 120000).

harvest_urls() ->
    harvest_urls(?NUM_REQUEST_THREADS, []).

harvest_urls(Num_Processes, [Link|Rest]) ->
    io:format("Processes left to spawn: ~w~n", [Num_Processes]),
    case Num_Processes =< 0 of
        true ->
            receive
                {more_urls, More_Urls} ->
                    harvest_urls(Num_Processes, [Link|Rest] ++ More_Urls);
                completed ->
                    harvest_urls(Num_Processes + 1, [Link|Rest]);
                shutdown ->
                    ok
            after ?DEFAULT_TIMEOUT ->
                harvest_urls()
            end;
        false ->
            Url = erlang:binary_to_list(object:get_value(Link, "destination")),
            io:format("Parsing: ~s~n", [Url]),
            spawn(es_parse, start, [self(), Url]),
            harvest_urls(Num_Processes-1, Rest)
    end;

harvest_urls(Num_Processes, []) ->
    %%Urls = es_db:get_oldest_links(?NUM_REQUEST_THREADS div 2),
    %%harvest_urls(Num_Processes, Urls).
    receive
        {more_urls, More_Urls} ->
            harvest_urls(Num_Processes, More_Urls);
        completed ->
            harvest_urls(Num_Processes + 1, []);
        shutdown ->
            ok
    end.

get_more_urls(Pid) ->
    receive
        shutdown ->
            ok
    after 1 ->
        Urls = es_db:get_oldest_links(?NUM_REQUEST_THREADS * 2),
        io:format("Got Urls!~n", []),
        Pid ! {more_urls, Urls},
        get_more_urls(Pid)
    end.

start() ->
    inets:start(),
    Pid = spawn(?MODULE, harvest_urls, []),
    Mon = erlang:monitor(process, Pid),
    Pid2 = spawn(?MODULE, get_more_urls, [Pid]),
    Mon2 = erlang:monitor(process, Pid2),
    receive
        {'DOWN', Mon, Type, Object, Info} ->
            Pid2 ! shutdown;
        {'DOWN', Mon2, Type, Object, Info} ->
            Pid ! shutdown
    end,
    start().

test() ->
    start(),
    ok.